{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['KMP_DUPLICATE_LIB_OK']='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import time\n",
    "from datetime import timedelta\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Untitled.ipynb         \u001b[31msample_submission.csv\u001b[m\u001b[m* \u001b[31mtrain.csv\u001b[m\u001b[m*\r\n",
      "result_v1.csv          \u001b[31mtest.csv\u001b[m\u001b[m*\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读入csv文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(42000, 785)\n"
     ]
    }
   ],
   "source": [
    "print(data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>pixel0</th>\n",
       "      <th>pixel1</th>\n",
       "      <th>pixel2</th>\n",
       "      <th>pixel3</th>\n",
       "      <th>pixel4</th>\n",
       "      <th>pixel5</th>\n",
       "      <th>pixel6</th>\n",
       "      <th>pixel7</th>\n",
       "      <th>pixel8</th>\n",
       "      <th>...</th>\n",
       "      <th>pixel774</th>\n",
       "      <th>pixel775</th>\n",
       "      <th>pixel776</th>\n",
       "      <th>pixel777</th>\n",
       "      <th>pixel778</th>\n",
       "      <th>pixel779</th>\n",
       "      <th>pixel780</th>\n",
       "      <th>pixel781</th>\n",
       "      <th>pixel782</th>\n",
       "      <th>pixel783</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 785 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  \\\n",
       "0      1       0       0       0       0       0       0       0       0   \n",
       "1      0       0       0       0       0       0       0       0       0   \n",
       "2      1       0       0       0       0       0       0       0       0   \n",
       "3      4       0       0       0       0       0       0       0       0   \n",
       "4      0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "   pixel8    ...     pixel774  pixel775  pixel776  pixel777  pixel778  \\\n",
       "0       0    ...            0         0         0         0         0   \n",
       "1       0    ...            0         0         0         0         0   \n",
       "2       0    ...            0         0         0         0         0   \n",
       "3       0    ...            0         0         0         0         0   \n",
       "4       0    ...            0         0         0         0         0   \n",
       "\n",
       "   pixel779  pixel780  pixel781  pixel782  pixel783  \n",
       "0         0         0         0         0         0  \n",
       "1         0         0         0         0         0  \n",
       "2         0         0         0         0         0  \n",
       "3         0         0         0         0         0  \n",
       "4         0         0         0         0         0  \n",
       "\n",
       "[5 rows x 785 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据清洗"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "输入数据 => 二维数组"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "images = data.iloc[:, 1:].values\n",
    "\n",
    "images = images.astype(np.float)\n",
    "\n",
    "images = np.multiply(images, 1.0/255.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s = data['label'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "大多数多分类问题\n",
    "\n",
    "标签方法： one-hot向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def dense_to_one_hot(s_dense, num_classes):\n",
    "    num_s = s_dense.shape[0]\n",
    "    index_offset = np.arange(num_s) * num_classes\n",
    "    s_one_hot = np.zeros((num_s, num_classes))\n",
    "    s_one_hot.flat[index_offset + s_dense.ravel()] = 1\n",
    "    return s_one_hot\n",
    "\n",
    "s = dense_to_one_hot(s, 10)\n",
    "s = s.astype(np.uint8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 设置基础变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "img_size_flat = images.shape[1]\n",
    "\n",
    "img_size = np.ceil(np.sqrt(img_size_flat)).astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "num_channels = 1\n",
    "num_classes = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 划分训练集+测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "VALIDATION_SIZE = 2000\n",
    "validation_images = images[:VALIDATION_SIZE]\n",
    "validation_labels = s[:VALIDATION_SIZE]\n",
    "\n",
    "train_images = images[VALIDATION_SIZE:]\n",
    "train_labels = s[VALIDATION_SIZE:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "validation_lables_cls = data['label'].values[:VALIDATION_SIZE]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 784)\n",
      "(2000, 784)\n"
     ]
    }
   ],
   "source": [
    "print(train_images.shape)\n",
    "print(validation_images.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorflow Graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 神经网络——函数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 神经网路参数设置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 第一层卷积层\n",
    "filter_size1 = 5\n",
    "num_filters1 = 16\n",
    "\n",
    "# 第二层卷积层\n",
    "filter_size2 = 5\n",
    "num_filters2 = 36\n",
    "\n",
    "# 全连接层\n",
    "fc_size = 128  # 全连接层结点数量"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 创建权重和偏差变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def new_weights(shape):\n",
    "    return tf.Variable(tf.truncated_normal(shape, stddev=0.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def new_biases(length):\n",
    "    return tf.Variable(tf.constant(0.05, shape=[length]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 创建卷积层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def new_conv_layer(input, \n",
    "                  num_input_channels,\n",
    "                  filter_size,\n",
    "                  num_filters,\n",
    "                  use_pooling=True):\n",
    "    \n",
    "    # 滤波器的形状设置\n",
    "    shape = [filter_size, filter_size, num_input_channels, num_filters]\n",
    "    \n",
    "    # 根据滤波器的形状设置权重\n",
    "    weights = new_weights(shape=shape)\n",
    "    \n",
    "    # 根据滤波器的数量设置偏差\n",
    "    biases = new_biases(length=num_filters)\n",
    "    \n",
    "    # 创建新的tensorflow卷积层\n",
    "    # strides参数：图像数量、X轴stride、Y轴stride、输入的通道数\n",
    "    # padding参数：SAME=padded with zeroes\n",
    "    layer = tf.nn.conv2d(input=input,\n",
    "                        filter=weights,\n",
    "                        strides=[1, 1, 1, 1], \n",
    "                        padding='SAME')\n",
    "    \n",
    "    layer += biases\n",
    "    \n",
    "    if use_pooling:\n",
    "        \n",
    "        layer = tf.nn.max_pool(value=layer,\n",
    "                              ksize=[1, 2, 2, 1],\n",
    "                              strides=[1, 2, 2 ,1],\n",
    "                              padding='SAME')\n",
    "    # ReLU=max(x, 0)\n",
    "    layer = tf.nn.relu(layer)\n",
    "    \n",
    "    # 返回结果层和权重\n",
    "    return layer, weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 卷积层转为全连接层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def flatten_layer(layer):\n",
    "    # 卷积层的形状\n",
    "    layer_shape = layer.get_shape()\n",
    "\n",
    "    # layer_shape == [num_images, img_height, img_width, num_channels]\n",
    "\n",
    "    # The number of features is: img_height * img_width * num_channels\n",
    "    # We can use a function from TensorFlow to calculate this.\n",
    "    # 特征的数量\n",
    "    num_features = layer_shape[1:4].num_elements()\n",
    "    \n",
    "    # Reshape the layer to [num_images, num_features].\n",
    "    # Note that we just set the size of the second dimension\n",
    "    # to num_features and the size of the first dimension to -1\n",
    "    # which means the size in that dimension is calculated\n",
    "    # so the total size of the tensor is unchanged from the reshaping.\n",
    "    # 如果第一个被设置为-1，图像数量大小会被自动计算\n",
    "    layer_flat = tf.reshape(layer, [-1, num_features])\n",
    "\n",
    "    # The shape of the flattened layer is now:\n",
    "    # [num_images, img_height * img_width * num_channels]\n",
    "\n",
    "    # Return both the flattened layer and the number of features.\n",
    "    return layer_flat, num_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 创建全连接层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def new_fc_layer(input, \n",
    "                num_inputs,\n",
    "                num_outputs,\n",
    "                use_relu=True):\n",
    "    \n",
    "    weights = new_weights(shape=[num_inputs, num_outputs])\n",
    "    biases = new_biases(length=num_outputs)\n",
    "    \n",
    "    layer = tf.matmul(input, weights) + biases\n",
    "    \n",
    "    if use_relu:\n",
    "        layer = tf.nn.relu(layer)\n",
    "    return layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 神经网络——变量和层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, shape=[None, img_size_flat], name='x')\n",
    "\n",
    "x_image = tf.reshape(x, [-1, img_size, img_size, num_channels])\n",
    "\n",
    "y_true = tf.placeholder(tf.float32, shape=[None, num_classes], name='y')\n",
    "\n",
    "y_true_cls = tf.argmax(y_true, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "layer_conv1, weights_conv1 = \\\n",
    "    new_conv_layer(input=x_image,\n",
    "                   num_input_channels=num_channels,\n",
    "                   filter_size=filter_size1,\n",
    "                   num_filters=num_filters1,\n",
    "                   use_pooling=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Relu:0' shape=(?, 14, 14, 16) dtype=float32>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_conv1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "layer_conv2, weights_conv2 = \\\n",
    "    new_conv_layer(input=layer_conv1,\n",
    "                  num_input_channels=num_filters1,\n",
    "                  filter_size=filter_size2,\n",
    "                  num_filters=num_filters2,\n",
    "                  use_pooling=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Relu_1:0' shape=(?, 7, 7, 36) dtype=float32>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_conv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "layer_flat, num_features = flatten_layer(layer_conv2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Reshape_1:0' shape=(?, 1764) dtype=float32>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_flat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1764"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "layer_fc1 = new_fc_layer(input=layer_flat,\n",
    "                        num_inputs=num_features,\n",
    "                        num_outputs=fc_size,\n",
    "                        use_relu=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Relu_2:0' shape=(?, 128) dtype=float32>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_fc1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "layer_fc2 = new_fc_layer(input=layer_fc1,\n",
    "                        num_inputs=fc_size,\n",
    "                        num_outputs=num_classes,\n",
    "                        use_relu=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'add_3:0' shape=(?, 10) dtype=float32>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_fc2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = tf.nn.softmax(layer_fc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred_cls = tf.argmax(y_pred, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 神经网络——损失及优化方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-37-0ed320d02193>:2: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=layer_fc2,\n",
    "                                                     labels=y_true)\n",
    "cost = tf.reduce_mean(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LEARNING_RATE = 1e-4\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE).minimize(cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 神经网络——性能度量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(y_pred_cls, y_true_cls)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorflow——开始运行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "session = tf.Session()\n",
    "session.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## feed，训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用全局变量，灵活控制训练次数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_mini_batch = 128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "mini_batch随机选择样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gen_random_batch(train_images, train_labels, batch_size=100):\n",
    "    \n",
    "    # 多个随机数\n",
    "    idx = np.random.randint(low=0, high=len(train_images), size=batch_size)\n",
    "    \n",
    "    # 根据索引获取值\n",
    "    x_batch = train_images[idx]\n",
    "    y_batch = train_labels[idx]\n",
    "    \n",
    "    return x_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "total_iterations = 0\n",
    "\n",
    "def optimize(num_iterations):\n",
    "    \n",
    "    global total_iterations\n",
    "    \n",
    "    start_time = time.time()\n",
    "    \n",
    "    for i in range(total_iterations, total_iterations+num_iterations):\n",
    "        \n",
    "        # 训练数据中随机选择mini_batch\n",
    "        x_batch, y_true_batch = gen_random_batch(train_images=train_images, \n",
    "                                                train_labels=train_labels,\n",
    "                                                batch_size=train_mini_batch)\n",
    "        \n",
    "        feed_dict_train = {\n",
    "            x:x_batch,\n",
    "            y_true:y_true_batch\n",
    "        }\n",
    "        \n",
    "        session.run(optimizer, feed_dict=feed_dict_train)\n",
    "        \n",
    "        # 训练时同步准确率\n",
    "        if i%100 == 0:\n",
    "            \n",
    "            acc = session.run(accuracy, feed_dict=feed_dict_train)\n",
    "            msg = \"优化迭代: {0}, 训练准确度为:{1}\"\n",
    "            print(msg.format((i+1), acc))\n",
    "            \n",
    "    total_iterations += num_iterations\n",
    "    \n",
    "    end_time = time.time()\n",
    "    \n",
    "    time_diff = end_time - start_time\n",
    "    print(\"训练耗时: {0}\".format(timedelta(seconds=int(time_diff))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试集评估函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def print_test_acc(test_images, test_labels):\n",
    "    \n",
    "    x_batch, y_true_batch = test_images, test_labels\n",
    "    \n",
    "    feed_dict_test = {\n",
    "        x: x_batch,\n",
    "        y_true:y_true_batch\n",
    "    }\n",
    "    \n",
    "    acc = session.run(accuracy, feed_dict=feed_dict_test)\n",
    "    print(\"测试集准确率为: {0}\".format(acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 正式开始"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'optimize' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-ece948985ba6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_iterations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'optimize' is not defined"
     ]
    }
   ],
   "source": [
    "optimize(num_iterations=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试集评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "测试集准确率为: 0.9470000267028809\n"
     ]
    }
   ],
   "source": [
    "print_test_acc(validation_images, validation_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 识别"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Untitled.ipynb         \u001b[31msample_submission.csv\u001b[m\u001b[m* \u001b[31mtrain.csv\u001b[m\u001b[m*\r\n",
      "result_v1.csv          \u001b[31mtest.csv\u001b[m\u001b[m*\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_data = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "images_test = test_data.values\n",
    "\n",
    "images_test = images_test.astype(np.float)\n",
    "\n",
    "images_test = np.multiply(images_test, 1.0/255.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28000, 784)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "feed_dict_test = {\n",
    "    x:images_test\n",
    "}\n",
    "\n",
    "pred_test = session.run(y_pred_cls, feed_dict=feed_dict_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 0, 9, ..., 3, 9, 2])"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 输出结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_sample = pd.read_csv('sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ImageId</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ImageId  Label\n",
       "0        1      0\n",
       "1        2      0"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sample.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_len = len(pred_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result_df = pd.DataFrame({'ImageId':range(1,test_len+1), \n",
    "                         'Label':pred_test}, index=range(0, test_len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "result_df.to_csv('result_v1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py3.6]",
   "language": "python",
   "name": "conda-env-py3.6-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "200px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
